{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import DRL.takeoff_dqn as tfdqn\n",
    "import LLCsimple\n",
    "import LLC \n",
    "import scaffold.fgdata as dfer\n",
    "import scaffold.pidpilot as PID\n",
    "import datetime\n",
    "import fgmodule.fgenv as fgenv\n",
    "import pandas as pd\n",
    "import os\n",
    "from scaffold.timer import gettime\n",
    "\n",
    "'''\n",
    "##########controlframe############\n",
    "\n",
    "\n",
    "##########state_dict##############\n",
    "{'aileron': 0.0, 'elevator': 0.0, 'rudder': -0.026, 'flaps': 0.0, 'throttle0': 0.6, 'throttle1': 0.6, 'vsi-fpm': 0.0, 'alt-ft': -372.808502, 'ai-pitch': 0.401045, 'ai-roll': 0.050598, 'ai-offset': 0.0, 'hi-heading': 80.568947, 'roll-deg': 0.050616, 'pitch-deg': 0.401055, 'heading-deg': 90.013458, 'airspeed-kt': 71.631187, 'speed-north-fps': -0.021637, 'speed-east-fps': 119.609383, 'speed-down-fps': -0.071344, 'uBody-fps': 119.606964, 'vBody-fps': -0.005778, 'wBody-fps': 0.765776, 'north-accel-fps_sec': 0.118887, 'east-accel-fps_sec': 5.870498, 'down-accel-fps_sec': -0.003219, 'x-accel-fps_sec': 6.095403, 'y-accel-fps_sec': -0.148636, 'z-accel-fps_sec': -32.113453, 'latitude': 21.325245, 'longitude': -157.93947, 'altitude': 22.297876, 'crashed': 0.0}\n",
    "'''\n",
    "\n",
    "LLC_FEATURE_BOUNDS = {\n",
    "    'aileron': [-1, 1],  # 副翼 控制飞机翻滚 [-1,1] left/right\n",
    "    'rudder': [-1, 1],  # 方向舵 控制飞机转弯（地面飞机方向控制） 0 /enter\n",
    "    'pitch-deg': [-90., 90.],  # 飞机俯仰角\n",
    "    'roll-deg': [-180., 180.],  # 飞机滚转角\n",
    "    'heading-deg': [0., 360.],  # 飞机朝向\n",
    "}\n",
    "LLC_GOAL_BOUNDS = {\n",
    "    'pitch-deg': [-90., 90.],  # 飞机俯仰角\n",
    "    'roll-deg': [-180., 180.],  # 飞机滚转角\n",
    "    'heading-deg': [0., 360.],  # 飞机朝向\n",
    "}\n",
    "goals = {\n",
    "    'pitch-deg': 0.,  # 飞机俯仰角\n",
    "    'roll-deg': 0.,  # 飞机滚转角\n",
    "    'heading-deg': 90.,  # 飞机朝向\n",
    "}\n",
    "\n",
    "LLC_ACTION_BOUNDS = {\n",
    "    'aileron': [-1, 1],  # 副翼 控制飞机翻滚 [-1,1] left/right\n",
    "    'rudder': [-1, 1],  # 方向舵 控制飞机转弯（地面飞机方向控制） 0 /enter\n",
    "}\n",
    "\n",
    "def fgstart(fg2client_addr=(\"127.0.0.1\", 5700), client2fg_addr=(\"127.0.0.1\", 5701), telnet_addr=(\"127.0.0.1\", 5555)):\n",
    "    '''\n",
    "    简化FG通信启动过程\n",
    "    ---\n",
    "    Inputs:\n",
    "        - fg2client_addr\n",
    "        - client2fg_addr\n",
    "        - telnet_addr\n",
    "    Outputs:\n",
    "        - myfgenv:初始话完成的fgenv类\n",
    "\n",
    "    '''\n",
    "    myfgenv = fgenv.fgenv(telnet_addr, fg2client_addr, client2fg_addr)\n",
    "    # initial_state = myfgenv.initial()\n",
    "    myfgenv.initial()\n",
    "\n",
    "    return myfgenv\n",
    "\n",
    "\n",
    "MAX_EPISODE = 1000\n",
    "MAX_EP_STEPS = 200\n",
    "\n",
    "GAMMA = 0.9\n",
    "LR_A = 0.001    # learning rate for actor\n",
    "LR_C = 0.01     # learning rate for critic\n",
    "\n",
    "##\n",
    "epoch = 1000\n",
    "step = 220"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flylog saving path check pass!\n",
      "send data to 127.0.0.1:5701 !\n",
      "fgudp is ready!!!\n",
      "Bind UDP on 127.0.0.1:5700 !\n",
      "save log to data/flylog\n",
      "auto start完成 16 : 16 : 38\n",
      "fgudp stop begin! waiting for 2 thread quit\n",
      "send data to 127.0.0.1:5701 stop!UDP rece on 127.0.0.1:5700 stop!\n",
      "\n",
      "reset完成 16 : 16 : 39\n",
      "flylog saving path check pass!\n",
      "send data to 127.0.0.1:5701 !\n",
      "fgudp is ready!!!\n",
      "Bind UDP on 127.0.0.1:5700 !\n",
      "save log to data/flylog\n",
      "auto start完成 16 : 17 : 34\n"
     ]
    }
   ],
   "source": [
    "myfgenv = fgstart()\n",
    "\n",
    "# bounds = {\n",
    "#     'rudder': [-1, 1],  # 方向舵 控制飞机转弯（地面飞机方向控制） 0 /enter\n",
    "# }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "myllc = LLCsimple.LLC(LLC_FEATURE_BOUNDS,LLC_GOAL_BOUNDS,LLC_ACTION_BOUNDS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replay to sky 完成!time:16 : 18 : 3\n",
      "auto start完成 16 : 18 : 6\n"
     ]
    }
   ],
   "source": [
    "state = myfgenv.replay(\"sky\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "state = {'aileron': 0.0,\n",
    " 'elevator': 0.0,\n",
    " 'rudder': 0.0,\n",
    " 'flaps': 0.0,\n",
    " 'throttle0': 0.0,\n",
    " 'throttle1': 0.0,\n",
    " 'vsi-fpm': 0.0,\n",
    " 'alt-ft': -370.319122,\n",
    " 'ai-pitch': 0.534232,\n",
    " 'ai-roll': 0.277171,\n",
    " 'ai-offset': 0.0,\n",
    " 'hi-heading': 83.792358,\n",
    " 'roll-deg': 0.056131,\n",
    " 'pitch-deg': 0.467942,\n",
    " 'heading-deg': 90.00074,\n",
    " 'airspeed-kt': 1.532065,\n",
    " 'speed-north-fps': -0.000281,\n",
    " 'speed-east-fps': -0.007067,\n",
    " 'speed-down-fps': 0.000105,\n",
    " 'uBody-fps': -0.007067,\n",
    " 'vBody-fps': 0.000281,\n",
    " 'wBody-fps': 4.7e-05,\n",
    " 'north-accel-fps_sec': -1.4e-05,\n",
    " 'east-accel-fps_sec': -0.009769,\n",
    " 'down-accel-fps_sec': -0.000142,\n",
    " 'x-accel-fps_sec': 0.252822,\n",
    " 'y-accel-fps_sec': -0.031483,\n",
    " 'z-accel-fps_sec': -32.151367,\n",
    " 'latitude': 21.325247,\n",
    " 'longitude': -157.943137,\n",
    " 'altitude': 21.358186,\n",
    " 'crashed': 0.0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "goal = goals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'pitch-deg': [-90.0, 90.0],\n",
       " 'roll-deg': [-180.0, 180.0],\n",
       " 'heading-deg': [0.0, 360.0]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myllc.goals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'aileron': 0.0,\n",
       " 'elevator': 0.0,\n",
       " 'rudder': 0.0,\n",
       " 'flaps': 0.0,\n",
       " 'throttle0': 0.21,\n",
       " 'throttle1': 0.21,\n",
       " 'vsi-fpm': 0.0,\n",
       " 'alt-ft': -373.07193,\n",
       " 'ai-pitch': 0.444673,\n",
       " 'ai-roll': 0.054528,\n",
       " 'ai-offset': 0.0,\n",
       " 'hi-heading': 80.698265,\n",
       " 'roll-deg': 0.054527,\n",
       " 'pitch-deg': 0.444676,\n",
       " 'heading-deg': 90.142323,\n",
       " 'airspeed-kt': 17.874088,\n",
       " 'speed-north-fps': -0.068237,\n",
       " 'speed-east-fps': 27.643753,\n",
       " 'speed-down-fps': -0.025079,\n",
       " 'uBody-fps': 27.643202,\n",
       " 'vBody-fps': -0.000252,\n",
       " 'wBody-fps': 0.189458,\n",
       " 'north-accel-fps_sec': -0.014251,\n",
       " 'east-accel-fps_sec': 1.868108,\n",
       " 'down-accel-fps_sec': -0.001648,\n",
       " 'x-accel-fps_sec': 2.117626,\n",
       " 'y-accel-fps_sec': -0.020975,\n",
       " 'z-accel-fps_sec': -32.138405,\n",
       " 'latitude': 21.325246,\n",
       " 'longitude': -157.94253,\n",
       " 'altitude': 21.546366,\n",
       " 'crashed': 0.0}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.49682351424715865\n",
      "[-0.5381245  -0.25378835]\n",
      "[-0.64095265  0.4908171 ]\n"
     ]
    }
   ],
   "source": [
    "action ,_ = myllc.choose_action(state,goal)\n",
    "print(LLCsimple.llc_reward(state , goal, old_action, action, 0))\n",
    "print(action)\n",
    "print(old_action)\n",
    "old_action = action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'LLCsimple' from 'C:\\\\Users\\\\niebu\\\\Desktop\\\\FlightGear\\\\FGAutopilot\\\\FGAutopilot\\\\LLCsimple.py'>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import importlib\n",
    "importlib.reload(LLCsimple)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "action = np.array([0.,0.])\n",
    "old_action = np.array([0.,0.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replay to sky 完成!time:21 : 27 : 16\n",
      "auto start完成 21 : 27 : 19\n",
      "Episode: 0  Reward: -238 Explore: 0.04\n",
      "replay to sky 完成!time:21 : 27 : 46\n",
      "auto start完成 21 : 27 : 49\n",
      "Episode: 1  Reward: -234 Explore: 0.04\n",
      "replay to sky 完成!time:21 : 28 : 16\n",
      "auto start完成 21 : 28 : 19\n",
      "Episode: 2  Reward: -243 Explore: 0.04\n",
      "replay to sky 完成!time:21 : 28 : 46\n",
      "auto start完成 21 : 28 : 49\n",
      "Episode: 3  Reward: -238 Explore: 0.04\n",
      "replay to sky 完成!time:21 : 29 : 16\n",
      "auto start完成 21 : 29 : 19\n",
      "Episode: 4  Reward: -245 Explore: 0.04\n",
      "replay to sky 完成!time:21 : 29 : 45\n",
      "auto start完成 21 : 29 : 48\n",
      "Episode: 5  Reward: -240 Explore: 0.04\n",
      "replay to sky 完成!time:21 : 30 : 15\n",
      "auto start完成 21 : 30 : 18\n",
      "Episode: 6  Reward: -239 Explore: 0.04\n",
      "replay to sky 完成!time:21 : 30 : 45\n",
      "auto start完成 21 : 30 : 48\n",
      "Episode: 7  Reward: -250 Explore: 0.04\n",
      "replay to sky 完成!time:21 : 31 : 15\n",
      "auto start完成 21 : 31 : 18\n",
      "Episode: 8  Reward: -241 Explore: 0.04\n",
      "replay to sky 完成!time:21 : 31 : 45\n",
      "auto start完成 21 : 31 : 48\n",
      "Episode: 9  Reward: -238 Explore: 0.04\n",
      "replay to sky 完成!time:21 : 32 : 15\n",
      "auto start完成 21 : 32 : 18\n",
      "Episode: 10  Reward: -241 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 32 : 46\n",
      "auto start完成 21 : 32 : 49\n",
      "Episode: 11  Reward: -247 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 33 : 16\n",
      "auto start完成 21 : 33 : 19\n",
      "Episode: 12  Reward: -240 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 33 : 46\n",
      "auto start完成 21 : 33 : 49\n",
      "Episode: 13  Reward: -235 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 34 : 16\n",
      "auto start完成 21 : 34 : 19\n",
      "Episode: 14  Reward: -250 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 34 : 46\n",
      "auto start完成 21 : 34 : 49\n",
      "Episode: 15  Reward: -255 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 35 : 16\n",
      "auto start完成 21 : 35 : 19\n",
      "Episode: 16  Reward: -237 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 35 : 46\n",
      "auto start完成 21 : 35 : 49\n",
      "Episode: 17  Reward: -240 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 36 : 16\n",
      "auto start完成 21 : 36 : 19\n",
      "Episode: 18  Reward: -235 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 36 : 45\n",
      "auto start完成 21 : 36 : 48\n",
      "Episode: 19  Reward: -249 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 37 : 15\n",
      "auto start完成 21 : 37 : 18\n",
      "Episode: 20  Reward: -245 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 37 : 46\n",
      "auto start完成 21 : 37 : 49\n",
      "Episode: 21  Reward: -253 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 38 : 16\n",
      "auto start完成 21 : 38 : 19\n",
      "Episode: 22  Reward: -244 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 38 : 46\n",
      "auto start完成 21 : 38 : 49\n",
      "Episode: 23  Reward: -241 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 39 : 16\n",
      "auto start完成 21 : 39 : 19\n",
      "Episode: 24  Reward: -233 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 39 : 46\n",
      "auto start完成 21 : 39 : 49\n",
      "Episode: 25  Reward: -246 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 40 : 15\n",
      "auto start完成 21 : 40 : 18\n",
      "Episode: 26  Reward: -250 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 40 : 46\n",
      "auto start完成 21 : 40 : 49\n",
      "Episode: 27  Reward: -244 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 41 : 16\n",
      "auto start完成 21 : 41 : 19\n",
      "Episode: 28  Reward: -247 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 41 : 46\n",
      "auto start完成 21 : 41 : 49\n",
      "Episode: 29  Reward: -252 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 42 : 16\n",
      "auto start完成 21 : 42 : 19\n",
      "Episode: 30  Reward: -245 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 42 : 46\n",
      "auto start完成 21 : 42 : 49\n",
      "Episode: 31  Reward: -248 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 43 : 16\n",
      "auto start完成 21 : 43 : 19\n",
      "Episode: 32  Reward: -238 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 43 : 46\n",
      "auto start完成 21 : 43 : 49\n",
      "Episode: 33  Reward: -233 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 44 : 16\n",
      "auto start完成 21 : 44 : 19\n",
      "Episode: 34  Reward: -241 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 44 : 46\n",
      "auto start完成 21 : 44 : 49\n",
      "Episode: 35  Reward: -254 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 45 : 16\n",
      "auto start完成 21 : 45 : 19\n",
      "Episode: 36  Reward: -255 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 45 : 46\n",
      "auto start完成 21 : 45 : 49\n",
      "Episode: 37  Reward: -251 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 46 : 16\n",
      "auto start完成 21 : 46 : 19\n",
      "Episode: 38  Reward: -250 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 46 : 46\n",
      "auto start完成 21 : 46 : 49\n",
      "Episode: 39  Reward: -246 Explore: 0.03\n",
      "replay to sky 完成!time:21 : 47 : 16\n",
      "auto start完成 21 : 47 : 19\n",
      "Episode: 40  Reward: -233 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 47 : 46\n",
      "auto start完成 21 : 47 : 49\n",
      "Episode: 41  Reward: -235 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 48 : 16\n",
      "auto start完成 21 : 48 : 19\n",
      "Episode: 42  Reward: -251 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 48 : 47\n",
      "auto start完成 21 : 48 : 50\n",
      "Episode: 43  Reward: -250 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 49 : 17\n",
      "auto start完成 21 : 49 : 20\n",
      "Episode: 44  Reward: -250 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 49 : 47\n",
      "auto start完成 21 : 49 : 50\n",
      "Episode: 45  Reward: -232 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 50 : 17\n",
      "auto start完成 21 : 50 : 20\n",
      "Episode: 46  Reward: -251 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 50 : 46\n",
      "auto start完成 21 : 50 : 49\n",
      "Episode: 47  Reward: -235 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 51 : 17\n",
      "auto start完成 21 : 51 : 20\n",
      "Episode: 48  Reward: -248 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 51 : 46\n",
      "auto start完成 21 : 51 : 49\n",
      "Episode: 49  Reward: -233 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 52 : 16\n",
      "auto start完成 21 : 52 : 19\n",
      "Episode: 50  Reward: -234 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 52 : 46\n",
      "auto start完成 21 : 52 : 49\n",
      "Episode: 51  Reward: -252 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 53 : 16\n",
      "auto start完成 21 : 53 : 19\n",
      "Episode: 52  Reward: -234 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 53 : 46\n",
      "auto start完成 21 : 53 : 49\n",
      "Episode: 53  Reward: -235 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 54 : 16\n",
      "auto start完成 21 : 54 : 19\n",
      "Episode: 54  Reward: -236 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 54 : 46\n",
      "auto start完成 21 : 54 : 49\n",
      "Episode: 55  Reward: -253 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 55 : 16\n",
      "auto start完成 21 : 55 : 19\n",
      "Episode: 56  Reward: -252 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 55 : 46\n",
      "auto start完成 21 : 55 : 49\n",
      "Episode: 57  Reward: -253 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 56 : 16\n",
      "auto start完成 21 : 56 : 19\n",
      "Episode: 58  Reward: -252 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 56 : 46\n",
      "auto start完成 21 : 56 : 49\n",
      "Episode: 59  Reward: -255 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 57 : 16\n",
      "auto start完成 21 : 57 : 19\n",
      "Episode: 60  Reward: -249 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 57 : 46\n",
      "auto start完成 21 : 57 : 49\n",
      "Episode: 61  Reward: -251 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 58 : 17\n",
      "auto start完成 21 : 58 : 20\n",
      "Episode: 62  Reward: -254 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 58 : 47\n",
      "auto start完成 21 : 58 : 50\n",
      "Episode: 63  Reward: -255 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 59 : 17\n",
      "auto start完成 21 : 59 : 20\n",
      "Episode: 64  Reward: -234 Explore: 0.02\n",
      "replay to sky 完成!time:21 : 59 : 47\n",
      "auto start完成 21 : 59 : 50\n",
      "Episode: 65  Reward: -252 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 0 : 17\n",
      "auto start完成 22 : 0 : 20\n",
      "Episode: 66  Reward: -250 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 0 : 47\n",
      "auto start完成 22 : 0 : 50\n",
      "Episode: 67  Reward: -253 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 1 : 17\n",
      "auto start完成 22 : 1 : 20\n",
      "Episode: 68  Reward: -252 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 1 : 47\n",
      "auto start完成 22 : 1 : 50\n",
      "Episode: 69  Reward: -232 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 2 : 17\n",
      "auto start完成 22 : 2 : 20\n",
      "Episode: 70  Reward: -233 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 2 : 47\n",
      "auto start完成 22 : 2 : 50\n",
      "Episode: 71  Reward: -231 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 3 : 17\n",
      "auto start完成 22 : 3 : 20\n",
      "Episode: 72  Reward: -262 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 3 : 47\n",
      "auto start完成 22 : 3 : 50\n",
      "Episode: 73  Reward: -263 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 4 : 17\n",
      "auto start完成 22 : 4 : 20\n",
      "Episode: 74  Reward: -256 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 4 : 47\n",
      "auto start完成 22 : 4 : 50\n",
      "Episode: 75  Reward: -225 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 5 : 17\n",
      "auto start完成 22 : 5 : 20\n",
      "Episode: 76  Reward: -251 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 5 : 47\n",
      "auto start完成 22 : 5 : 50\n",
      "Episode: 77  Reward: -241 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 6 : 17\n",
      "auto start完成 22 : 6 : 20\n",
      "Episode: 78  Reward: -263 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 6 : 47\n",
      "auto start完成 22 : 6 : 50\n",
      "Episode: 79  Reward: -227 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 7 : 17\n",
      "auto start完成 22 : 7 : 20\n",
      "Episode: 80  Reward: -222 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 7 : 47\n",
      "auto start完成 22 : 7 : 50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 81  Reward: -256 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 8 : 17\n",
      "auto start完成 22 : 8 : 20\n",
      "Episode: 82  Reward: -232 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 8 : 47\n",
      "auto start完成 22 : 8 : 50\n",
      "Episode: 83  Reward: -228 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 9 : 17\n",
      "auto start完成 22 : 9 : 20\n",
      "Episode: 84  Reward: -240 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 9 : 47\n",
      "auto start完成 22 : 9 : 50\n",
      "Episode: 85  Reward: -240 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 10 : 17\n",
      "auto start完成 22 : 10 : 20\n",
      "Episode: 86  Reward: -229 Explore: 0.02\n",
      "replay to sky 完成!time:22 : 10 : 47\n",
      "auto start完成 22 : 10 : 50\n",
      "Episode: 87  Reward: -262 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 11 : 17\n",
      "auto start完成 22 : 11 : 20\n",
      "Episode: 88  Reward: -239 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 11 : 48\n",
      "auto start完成 22 : 11 : 51\n",
      "Episode: 89  Reward: -210 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 12 : 18\n",
      "auto start完成 22 : 12 : 21\n",
      "Episode: 90  Reward: -237 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 12 : 48\n",
      "auto start完成 22 : 12 : 51\n",
      "Episode: 91  Reward: -262 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 13 : 18\n",
      "auto start完成 22 : 13 : 21\n",
      "Episode: 92  Reward: -252 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 13 : 48\n",
      "auto start完成 22 : 13 : 51\n",
      "Episode: 93  Reward: -215 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 14 : 18\n",
      "auto start完成 22 : 14 : 21\n",
      "Episode: 94  Reward: -231 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 14 : 48\n",
      "auto start完成 22 : 14 : 51\n",
      "Episode: 95  Reward: -262 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 15 : 18\n",
      "auto start完成 22 : 15 : 21\n",
      "Episode: 96  Reward: -244 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 15 : 48\n",
      "auto start完成 22 : 15 : 51\n",
      "Episode: 97  Reward: -260 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 16 : 18\n",
      "auto start完成 22 : 16 : 21\n",
      "Episode: 98  Reward: -262 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 16 : 48\n",
      "auto start完成 22 : 16 : 51\n",
      "Episode: 99  Reward: -241 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 17 : 18\n",
      "auto start完成 22 : 17 : 21\n",
      "Episode: 100  Reward: -218 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 17 : 48\n",
      "auto start完成 22 : 17 : 51\n",
      "Episode: 101  Reward: -237 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 18 : 18\n",
      "auto start完成 22 : 18 : 21\n",
      "Episode: 102  Reward: -241 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 18 : 48\n",
      "auto start完成 22 : 18 : 51\n",
      "Episode: 103  Reward: -263 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 19 : 18\n",
      "auto start完成 22 : 19 : 21\n",
      "Episode: 104  Reward: -257 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 19 : 48\n",
      "auto start完成 22 : 19 : 51\n",
      "Episode: 105  Reward: -232 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 20 : 18\n",
      "auto start完成 22 : 20 : 21\n",
      "Episode: 106  Reward: -229 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 20 : 48\n",
      "auto start完成 22 : 20 : 51\n",
      "Episode: 107  Reward: -257 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 21 : 18\n",
      "auto start完成 22 : 21 : 21\n",
      "Episode: 108  Reward: -256 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 21 : 48\n",
      "auto start完成 22 : 21 : 51\n",
      "Episode: 109  Reward: -255 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 22 : 18\n",
      "auto start完成 22 : 22 : 21\n",
      "Episode: 110  Reward: -262 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 22 : 48\n",
      "auto start完成 22 : 22 : 51\n",
      "Episode: 111  Reward: -263 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 23 : 18\n",
      "auto start完成 22 : 23 : 21\n",
      "Episode: 112  Reward: -239 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 23 : 48\n",
      "auto start完成 22 : 23 : 51\n",
      "Episode: 113  Reward: -242 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 24 : 18\n",
      "auto start完成 22 : 24 : 21\n",
      "Episode: 114  Reward: -250 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 24 : 48\n",
      "auto start完成 22 : 24 : 51\n",
      "Episode: 115  Reward: -255 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 25 : 18\n",
      "auto start完成 22 : 25 : 21\n",
      "Episode: 116  Reward: -236 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 25 : 48\n",
      "auto start完成 22 : 25 : 51\n",
      "Episode: 117  Reward: -240 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 26 : 18\n",
      "auto start完成 22 : 26 : 21\n",
      "Episode: 118  Reward: -239 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 26 : 48\n",
      "auto start完成 22 : 26 : 51\n",
      "Episode: 119  Reward: -250 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 27 : 18\n",
      "auto start完成 22 : 27 : 21\n",
      "Episode: 120  Reward: -236 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 27 : 48\n",
      "auto start完成 22 : 27 : 51\n",
      "Episode: 121  Reward: -257 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 28 : 18\n",
      "auto start完成 22 : 28 : 21\n",
      "Episode: 122  Reward: -261 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 28 : 48\n",
      "auto start完成 22 : 28 : 51\n",
      "Episode: 123  Reward: -228 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 29 : 18\n",
      "auto start完成 22 : 29 : 21\n",
      "Episode: 124  Reward: -241 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 29 : 48\n",
      "auto start完成 22 : 29 : 51\n",
      "Episode: 125  Reward: -263 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 30 : 18\n",
      "auto start完成 22 : 30 : 21\n",
      "Episode: 126  Reward: -243 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 30 : 49\n",
      "auto start完成 22 : 30 : 52\n",
      "Episode: 127  Reward: -257 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 31 : 19\n",
      "auto start完成 22 : 31 : 22\n",
      "Episode: 128  Reward: -226 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 31 : 49\n",
      "auto start完成 22 : 31 : 52\n",
      "Episode: 129  Reward: -237 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 32 : 19\n",
      "auto start完成 22 : 32 : 22\n",
      "Episode: 130  Reward: -259 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 32 : 49\n",
      "auto start完成 22 : 32 : 52\n",
      "Episode: 131  Reward: -226 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 33 : 19\n",
      "auto start完成 22 : 33 : 22\n",
      "Episode: 132  Reward: -256 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 33 : 49\n",
      "auto start完成 22 : 33 : 52\n",
      "Episode: 133  Reward: -214 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 34 : 19\n",
      "auto start完成 22 : 34 : 22\n",
      "Episode: 134  Reward: -242 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 34 : 49\n",
      "auto start完成 22 : 34 : 52\n",
      "Episode: 135  Reward: -232 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 35 : 19\n",
      "auto start完成 22 : 35 : 22\n",
      "Episode: 136  Reward: -226 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 35 : 49\n",
      "auto start完成 22 : 35 : 52\n",
      "Episode: 137  Reward: -255 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 36 : 19\n",
      "auto start完成 22 : 36 : 22\n",
      "Episode: 138  Reward: -238 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 36 : 49\n",
      "auto start完成 22 : 36 : 52\n",
      "Episode: 139  Reward: -219 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 37 : 19\n",
      "auto start完成 22 : 37 : 22\n",
      "Episode: 140  Reward: -249 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 37 : 49\n",
      "auto start完成 22 : 37 : 52\n",
      "Episode: 141  Reward: -217 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 38 : 19\n",
      "auto start完成 22 : 38 : 22\n",
      "Episode: 142  Reward: -252 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 38 : 49\n",
      "auto start完成 22 : 38 : 52\n",
      "Episode: 143  Reward: -237 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 39 : 19\n",
      "auto start完成 22 : 39 : 22\n",
      "Episode: 144  Reward: -238 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 39 : 50\n",
      "auto start完成 22 : 39 : 53\n",
      "Episode: 145  Reward: -212 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 40 : 20\n",
      "auto start完成 22 : 40 : 23\n",
      "Episode: 146  Reward: -258 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 40 : 50\n",
      "auto start完成 22 : 40 : 53\n",
      "Episode: 147  Reward: -241 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 41 : 20\n",
      "auto start完成 22 : 41 : 23\n",
      "Episode: 148  Reward: -232 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 41 : 50\n",
      "auto start完成 22 : 41 : 53\n",
      "Episode: 149  Reward: -235 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 42 : 20\n",
      "auto start完成 22 : 42 : 23\n",
      "Episode: 150  Reward: -220 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 42 : 50\n",
      "auto start完成 22 : 42 : 53\n",
      "Episode: 151  Reward: -253 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 43 : 20\n",
      "auto start完成 22 : 43 : 23\n",
      "Episode: 152  Reward: -241 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 43 : 50\n",
      "auto start完成 22 : 43 : 53\n",
      "Episode: 153  Reward: -256 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 44 : 20\n",
      "auto start完成 22 : 44 : 23\n",
      "Episode: 154  Reward: -225 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 44 : 50\n",
      "auto start完成 22 : 44 : 53\n",
      "Episode: 155  Reward: -258 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 45 : 20\n",
      "auto start完成 22 : 45 : 23\n",
      "Episode: 156  Reward: -212 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 45 : 50\n",
      "auto start完成 22 : 45 : 53\n",
      "Episode: 157  Reward: -226 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 46 : 20\n",
      "auto start完成 22 : 46 : 23\n",
      "Episode: 158  Reward: -258 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 46 : 50\n",
      "auto start完成 22 : 46 : 53\n",
      "Episode: 159  Reward: -258 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 47 : 20\n",
      "auto start完成 22 : 47 : 23\n",
      "Episode: 160  Reward: -262 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 47 : 50\n",
      "auto start完成 22 : 47 : 53\n",
      "Episode: 161  Reward: -254 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 48 : 20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auto start完成 22 : 48 : 23\n",
      "Episode: 162  Reward: -255 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 48 : 50\n",
      "auto start完成 22 : 48 : 53\n",
      "Episode: 163  Reward: -240 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 49 : 20\n",
      "auto start完成 22 : 49 : 23\n",
      "Episode: 164  Reward: -217 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 49 : 50\n",
      "auto start完成 22 : 49 : 53\n",
      "Episode: 165  Reward: -243 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 50 : 20\n",
      "auto start完成 22 : 50 : 23\n",
      "Episode: 166  Reward: -230 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 50 : 50\n",
      "auto start完成 22 : 50 : 53\n",
      "Episode: 167  Reward: -244 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 51 : 20\n",
      "auto start完成 22 : 51 : 23\n",
      "Episode: 168  Reward: -237 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 51 : 50\n",
      "auto start完成 22 : 51 : 53\n",
      "Episode: 169  Reward: -239 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 52 : 20\n",
      "auto start完成 22 : 52 : 23\n",
      "Episode: 170  Reward: -210 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 52 : 50\n",
      "auto start完成 22 : 52 : 53\n",
      "Episode: 171  Reward: -261 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 53 : 20\n",
      "auto start完成 22 : 53 : 23\n",
      "Episode: 172  Reward: -241 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 53 : 50\n",
      "auto start完成 22 : 53 : 53\n",
      "Episode: 173  Reward: -256 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 54 : 20\n",
      "auto start完成 22 : 54 : 23\n",
      "Episode: 174  Reward: -217 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 54 : 50\n",
      "auto start完成 22 : 54 : 53\n",
      "Episode: 175  Reward: -232 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 55 : 20\n",
      "auto start完成 22 : 55 : 23\n",
      "Episode: 176  Reward: -256 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 55 : 50\n",
      "auto start完成 22 : 55 : 53\n",
      "Episode: 177  Reward: -217 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 56 : 19\n",
      "auto start完成 22 : 56 : 22\n",
      "Episode: 178  Reward: -232 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 56 : 49\n",
      "auto start完成 22 : 56 : 53\n",
      "Episode: 179  Reward: -252 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 57 : 20\n",
      "auto start完成 22 : 57 : 23\n",
      "Episode: 180  Reward: -216 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 57 : 50\n",
      "auto start完成 22 : 57 : 53\n",
      "Episode: 181  Reward: -256 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 58 : 20\n",
      "auto start完成 22 : 58 : 23\n",
      "Episode: 182  Reward: -215 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 58 : 50\n",
      "auto start完成 22 : 58 : 53\n",
      "Episode: 183  Reward: -216 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 59 : 20\n",
      "auto start完成 22 : 59 : 23\n",
      "Episode: 184  Reward: -251 Explore: 0.01\n",
      "replay to sky 完成!time:22 : 59 : 50\n",
      "auto start完成 22 : 59 : 53\n",
      "Episode: 185  Reward: -230 Explore: 0.01\n",
      "replay to sky 完成!time:23 : 0 : 20\n",
      "auto start完成 23 : 0 : 23\n",
      "Episode: 186  Reward: -219 Explore: 0.01\n",
      "replay to sky 完成!time:23 : 0 : 50\n",
      "auto start完成 23 : 0 : 53\n",
      "Episode: 187  Reward: -256 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 1 : 20\n",
      "auto start完成 23 : 1 : 23\n",
      "Episode: 188  Reward: -259 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 1 : 50\n",
      "auto start完成 23 : 1 : 53\n",
      "Episode: 189  Reward: -215 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 2 : 19\n",
      "auto start完成 23 : 2 : 22\n",
      "Episode: 190  Reward: -239 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 2 : 50\n",
      "auto start完成 23 : 2 : 53\n",
      "Episode: 191  Reward: -213 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 3 : 20\n",
      "auto start完成 23 : 3 : 23\n",
      "Episode: 192  Reward: -263 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 3 : 50\n",
      "auto start完成 23 : 3 : 53\n",
      "Episode: 193  Reward: -264 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 4 : 19\n",
      "auto start完成 23 : 4 : 22\n",
      "Episode: 194  Reward: -264 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 4 : 48\n",
      "auto start完成 23 : 4 : 51\n",
      "Episode: 195  Reward: -235 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 5 : 17\n",
      "auto start完成 23 : 5 : 20\n",
      "Episode: 196  Reward: -264 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 5 : 47\n",
      "auto start完成 23 : 5 : 50\n",
      "Episode: 197  Reward: -252 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 6 : 16\n",
      "auto start完成 23 : 6 : 19\n",
      "Episode: 198  Reward: -263 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 6 : 46\n",
      "auto start完成 23 : 6 : 49\n",
      "Episode: 199  Reward: -263 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 7 : 16\n",
      "auto start完成 23 : 7 : 19\n",
      "Episode: 200  Reward: -260 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 7 : 46\n",
      "auto start完成 23 : 7 : 49\n",
      "Episode: 201  Reward: -262 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 8 : 16\n",
      "auto start完成 23 : 8 : 19\n",
      "Episode: 202  Reward: -258 Explore: 0.00\n",
      "replay to sky 完成!time:23 : 8 : 46\n",
      "auto start完成 23 : 8 : 49\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-14-627577236709>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     38\u001b[0m                \u001b[1;31m# if ep_reward > -300:RENDER = True\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m                \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m            \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0.1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    " for e in range(epoch):\n",
    "        state = myfgenv.replay(\"sky\")\n",
    "        time.sleep(2)\n",
    "\n",
    "        next_goal = goal\n",
    "\n",
    "        ep_reward = 0\n",
    "\n",
    "        for s in range(step):\n",
    "        \n",
    "            \n",
    "            old_action = action \n",
    "            action,action_true = myllc.choose_action(state,goal)\n",
    "            \n",
    "            elevator = -0.01*(0.0-state['pitch-deg'])\n",
    "\n",
    "            action_frame = dfer.action2frame((action_true[0],elevator,action_true[1],0.6,0.6))\n",
    "            \n",
    "            next_state, reward , done , info = myfgenv.step(action_frame) \n",
    "            \n",
    "            r_ = LLCsimple.llc_reward(state , goal,old_action,action ,reward)\n",
    "            \n",
    "            next_goal = goals\n",
    "            myllc.learn(state, goal, r_, action,next_state , next_goal)\n",
    "            \n",
    "            state = next_state\n",
    "            goal = next_goal\n",
    "\n",
    "            ep_reward += r_\n",
    "            if done:\n",
    "                print('Episode:', e, ' Reward: %i' %\n",
    "                      int(ep_reward), 'Explore: %.2f' % myllc.var, )\n",
    "                break\n",
    "            \n",
    "            if s == step-1:\n",
    "                print('Episode:', e, ' Reward: %i' %\n",
    "                      int(ep_reward), 'Explore: %.2f' % myllc.var, )\n",
    "                # if ep_reward > -300:RENDER = True\n",
    "                break\n",
    "            time.sleep(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "myllc.save(\"modelckpt/LLCsimple_DDPG/LLCsimple_DDPG.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25081"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myllc.ddpg.pointer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0027973283603017762\n"
     ]
    }
   ],
   "source": [
    "t1= time.clock()\n",
    "myllc.learn(state, goal, r_, action,next_state , next_goal)\n",
    "print(time.clock()-t1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
